import os
import json
import pandas as pd
import numpy as np
from datetime import datetime
import hashlib

# 配置路径
data_dir = '/data/GuoCu_data/raw_data/'
output_file = '/data/GuoCu_data/processed_data/feature/user/user_inter_click.csv'

# 存储所有IP地址（包括无点击用户）
all_ips = set()
# 存储有效点击用户数据
click_users = []

# 第一轮：收集所有IP地址
for root, dirs, files in os.walk(data_dir):
    for file in files:
        if file.endswith('.csv'):
            file_path = os.path.join(root, file)
            try:
                df = pd.read_csv(file_path, header=None, usecols=[4])
                all_ips.update(df[4].dropna().unique())
            except:
                continue

# 第二轮：处理点击事件
all_clicks = []  # 收集所有点击事件详情

for root, dirs, files in os.walk(data_dir):
    for file in files:
        if file.endswith('.csv'):
            file_path = os.path.join(root, file)
            try:
                df = pd.read_csv(file_path, header=None, usecols=[0, 1, 2, 4])
                df.columns = ['eventTime', 'eventType', 'extraParam', 'ip']
                
                # 筛选click事件
                df = df[df['eventType'] == 'click']
                
                # 解析JSON并过滤有效点击
                def is_valid_click(extra_param):
                    try:
                        data = json.loads(extra_param)
                        return isinstance(data, dict) and 'id' in data and 'father' in data
                    except:
                        return False
                
                df = df[df['extraParam'].apply(is_valid_click)]
                
                if not df.empty:
                    # 转换时间格式
                    df['timestamp'] = pd.to_datetime(df['eventTime'])
                    df['hour'] = df['timestamp'].dt.hour
                    
                    # 解析extraParam中的id
                    def extract_id(extra_param):
                        try:
                            data = json.loads(extra_param)
                            return data.get('id', -1)
                        except:
                            return -1
                    
                    df['item_id'] = df['extraParam'].apply(extract_id)
                    
                    # 收集所有点击事件详情
                    all_clicks.extend(df[['ip', 'timestamp', 'hour', 'item_id']].to_dict('records'))
            except Exception as e:
                print(f"Error processing {file_path}: {str(e)}")

# 全局按IP合并点击事件
click_users = []
if all_clicks:
    click_df = pd.DataFrame(all_clicks)
    # 按用户分组处理所有点击事件
    for ip, group in click_df.groupby('ip'):
        group = group.sort_values('timestamp')
        n_clicks = len(group)
        
        # 时间差统计
        time_diffs = {'min': np.nan, 'mean': np.nan, 'std': np.nan, 'max': np.nan}
        if n_clicks > 1:
            diffs = group['timestamp'].diff().dropna().dt.total_seconds()
            time_diffs = {
                'min': diffs.min(),
                'mean': diffs.mean(),
                'std': diffs.std(ddof=0),
                'max': diffs.max()
            }
        
        # 平均点击小时
        hour_mean = group['hour'].mean()
        
        # 获取最后一次点击时间
        last_click_time = group['timestamp'].iloc[-1] if n_clicks > 0 else np.nan
        last_click_time = last_click_time.strftime('%Y-%m-%dT%H:%M:%S')
        click_users.append({
            'ip': ip,
            'user_total_clicks': n_clicks,
            'user_click_ts_diff_min': time_diffs['min'],
            'user_click_ts_diff_mean': time_diffs['mean'],
            'user_click_ts_diff_std': time_diffs['std'],
            'user_click_ts_diff_max': time_diffs['max'],
            'user_click_ts_hour_mean': hour_mean,
            'user_last_click_time': last_click_time,  # 新增字段
        })
print(click_users[0]['user_last_click_time'])
# 创建包含所有用户的全量DataFrame
all_users_df = pd.DataFrame(list(all_ips), columns=['ip'])

# 合并点击数据
if click_users:
    click_df = pd.DataFrame(click_users)
    result_df = pd.merge(all_users_df, click_df, on='ip', how='left')
else:
    result_df = all_users_df
    # 添加空特征列
    for col in ['user_total_clicks', 'user_click_ts_diff_min', 'user_click_ts_diff_mean',
                'user_click_ts_diff_std', 'user_click_ts_diff_max', 'user_click_ts_hour_mean',
                'user_last_click_time']:  # 添加新字段
        result_df[col] = np.nan

# 填充缺失值（无点击用户）
result_df.fillna({
    'user_total_clicks': 0,
    'user_click_ts_diff_min': -1,
    'user_click_ts_diff_mean': -1,
    'user_click_ts_diff_std': -1,
    'user_click_ts_diff_max': -1,
    'user_click_ts_hour_mean': -1,
    'user_last_click_time': "0000-00-00T00:00:00",  # 时间类型的缺失值
}, inplace=True)

# 保存结果
result_df.to_csv(output_file, index=False)
print(f"Saved results for {len(result_df)} users (including {len(result_df)-len(click_users)} zero-click users)")